var Nedb = require('nedb'),
    request = require('request'),
    fs = require('fs'),
    http = require('http'),
    mkdirp = require('mkdirp'),
    path = require('path'),
    db = new Nedb({ filename: './data.db', autoload: true }),
    regRow = /(<tr>)(.*href.*)(.*?)(<\/tr>)/ig,
    regLink = /(.*href=")(.*?)(">.*)/ig,
    regDate = /(.*<td class="m">)(.*?)(<\/td>.*)/ig;

var urls = [
    'http://nl.alpinelinux.org/alpine/v3.3/',
    'http://dl-2.alpinelinux.org/alpine/v3.3/',
    'http://dl-3.alpinelinux.org/alpine/v3.3/',
    'http://dl-4.alpinelinux.org/alpine/v3.3/',
    'http://dl-5.alpinelinux.org/alpine/v3.3/',
    'http://dl-6.alpinelinux.org/alpine/v3.3/',
    'http://dl-8.alpinelinux.org/alpine/v3.3/'
];

var getRandomUrl = function () {
    var max = urls.length - 1,
        min = 0;
    var id = Math.floor(Math.random() * (max - min + 1)) + min;
    return urls[id];
};

var getFileAliveThreadCount = getFileThreadCount = 50;

var log = function (log) {
    console.log(JSON.stringify(log));
};

var getFileList = function (cat) {
    var baseUrl = 'http://dl-2.alpinelinux.org/alpine/v3.3/';
    var url = baseUrl + cat;

    var newCount = 0;
    log({
        action: 'get file index',
        url: url,
    });
    request.get(url, { timeout: 30000 }, function (err, res) {
        if (err) {
            setTimeout(function () {
                getFileList(cat);
            }, 1000);
            log({
                action: 'get file index',
                error: err
            });
            return;
        }

        if (res.statusCode != 200) {
            setTimeout(function () {
                getFileList(cat);
            }, 1000);
            log({
                action: 'get file index',
                error: res.statusCode
            });
            return;
        }
        var rows = res.body.match(regRow);
        if (!rows) {
            setTimeout(function () {
                getFileList(cat);
            }, 1000);
            log({
                action: 'get file index',
                error: 'no file'
            });
            return;
        }
        rows.forEach(function (row, index, array) {
            var link = row.replace(regLink, '$2');
            if (link == '../') {
                return;
            }
            var date = row.replace(regDate, '$2');
            db.update({ _id: cat + link, date: { $ne: date } },
                {
                    $set: {
                        date: date,
                        status: 1
                    }
                },
                { upsert: true, returnUpdatedDocs: true },
                function (err, num, doc) {
                    if (err) {
                        if (err.errorType != 'uniqueViolated') {
                            log({
                                action: 'get file index',
                                error: 'db error: ' + err
                            });
                        }
                    } else if (num) {
                        newCount++;
                    }

                    if (index === array.length - 1) {
                        log({
                            action: 'get file index',
                            url: url,
                            total: rows.length - 1,
                            new: newCount
                        });
                        setTimeout(function () {
                            getFileList(cat);
                        }, 600000);
                    }
                });
        });
    });
};

getFileList('main/x86_64/');
getFileList('community/x86_64/');

var download = function (url, dest, thread, cb) {
    dest = unescape(dest);
    mkdirp.sync(path.dirname(dest));
    var file = fs.createWriteStream(dest);
    file.on('finish', function () {
        file.close(cb);
    });
    file.on('error', function (err) {
        log({
            error: 'save file error: ' + err
        });
    });
    var request = http.get(url, function (response) {
        log({
            action: 'download',
            thread: thread,
            url: url,
            size: response.headers['content-length']
        });
        response.pipe(file);
    }).on('error', function (err) {
        log({
            action: 'download',
            thread: thread,
            url: url,
            error: 'download file error: ' + err
        });
        if (cb) cb(err.message);
    });
};

var getFile = function (index) {
    db.update({ status: 1 },
        { $set: { status: 3 } },
        { returnUpdatedDocs: true },
        function (err, num, doc) {
            if (err) {
                log({
                    action: 'get file',
                    error: 'db error: ' + err
                });
                return;
            }

            if (!doc) {
                getFileAliveThreadCount--;
                if (!getFileAliveThreadCount) {
                    setTimeout(function () {
                        startGetFile();
                    }, 60000);
                }
                return;
            }

            var url = getRandomUrl();

            var http = require('http');
            var fs = require('fs');

            download(url + doc._id, 'files/' + doc._id, index, function (err) {
                var status = 2;
                if (err) {
                    log({
                        action: 'get file',
                        thread: index,
                        error: 'download error: ' + err
                    });
                    status = 1;
                }
                db.update({ _id: doc._id },
                    { $set: { status: status } },
                    function (err) {
                        getFile(index);
                        if (err) {
                            log({
                                action: 'get file',
                                thread: index,
                                error: 'db error: ' + err
                            });
                            return;
                        }
                        if (status == 2) {
                            db.count({ status: { $ne: 2 } }, function (err, count) {
                                log({
                                    action: 'save file',
                                    url: url + doc._id,
                                    thread: index,
                                    remain: count
                                });
                                if (!getFileAliveThreadCount) {
                                    log({
                                        action: 'download finished'
                                    });
                                }
                            });
                        }
                    });
            });
        });
};

var startGetFile = function () {
    db.update({ status: { $in: [3, 1] } },
        { $set: { status: 1 } },
        { multi: true },
        function (err, num) {
            if (!num) {
                setTimeout(function () {
                    startGetFile();
                }, 60000);
                return;
            }
            else {
                log({
                    action: 'get remaining files in download queue',
                    count: num
                });
            }
            getFileAliveThreadCount = num > getFileThreadCount ? getFileThreadCount : num;
            for (var i = 0; i < getFileAliveThreadCount; i++) {
                getFile(i);
            }
        });
};

startGetFile();


var static = require('node-static'),
    port = 8098,
    http = require('http');

// config
var file = new static.Server('./files', {
    cache: 3600,
    gzip: true
});

// serve
http.createServer(function (request, response) {
    request.addListener('end', function () {
        file.serve(request, response);
    }).resume();
}).listen(port);
log({
    action: 'alpine repository server listen',
    port: port
});
